{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# See https://archive.ics.uci.edu/ml/datasets/BlogFeedback for a description of the variables.\n",
    "# The response variable is the last column, representing the number of blog posts.\n",
    "data = pd.read_csv(\"http://users.csc.calpoly.edu/~dsun09/blogData_train.csv\",\n",
    "                   header=None)\n",
    "data[list(range(279))] = normalize(data[list(range(279))], axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (Batch) Gradient Descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def lm_gd(xvars, yvar, data, alpha):\n",
    "    \"\"\"Uses (batch) gradient descent to calculate the linear regression coefficients.\n",
    "    \n",
    "    Args:\n",
    "      xvars: A list of column indices representing the predictor variables\n",
    "      yvar: A column index representing the response variable\n",
    "      data: A Pandas data frame\n",
    "      alpha: A float representing the learning rate\n",
    "    \n",
    "    Returns:\n",
    "      a list of the linear regression coefficients\n",
    "    \"\"\"\n",
    "    \n",
    "    # create X matrix with intercept\n",
    "    X = pd.concat([pd.Series(1, index=data.index), data[xvars]], axis=1)\n",
    "    y = data[yvar]\n",
    "    \n",
    "    # initialize beta to all zeros\n",
    "    beta_old = np.inf\n",
    "    beta = np.zeros(len(xvars) + 1)\n",
    "    \n",
    "    # gradient descent iterations\n",
    "    while sum((beta - beta_old) ** 2) > 1e-4:\n",
    "        beta_old = beta\n",
    "        # update beta using gradient descent\n",
    "        beta = beta_old\n",
    "        # print objective value (so that you can monitor convergence)\n",
    "        obj = 0\n",
    "        print(obj)\n",
    "    \n",
    "    return beta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "beta = lm_gd(list(range(60, 262)), 280, data, .00001)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check your answer against sklearn."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = LinearRegression()\n",
    "X = data[list(range(60, 262))]\n",
    "model.fit(X, data[280])\n",
    "\n",
    "# print objective value for optimal beta\n",
    "np.sum((data[280] - model.predict(X)) ** 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stochastic Gradient Descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lm_sgd(xvars, yvar, data, alpha):\n",
    "    \"\"\"Uses stochastic gradient descent to calculate the linear regression coefficients.\n",
    "    \n",
    "    Args:\n",
    "      xvars: A list of column indices representing the predictor variables\n",
    "      yvar: A column index representing the response variable\n",
    "      data: A Pandas data frame\n",
    "      alpha: A float representing the learning rate\n",
    "    \n",
    "    Returns:\n",
    "      a list of the linear regression coefficients\n",
    "    \"\"\"\n",
    "    \n",
    "    # create X matrix with intercept\n",
    "    X = pd.concat([pd.Series(1, index=data.index), data[xvars]], axis=1)\n",
    "    y = data[yvar]\n",
    "    \n",
    "    # initialize beta to all zeros\n",
    "    beta_old = np.inf\n",
    "    beta = np.zeros(len(xvars) + 1)\n",
    "    \n",
    "    # update gradient, once for each row in the data\n",
    "    for i in len(X):\n",
    "        beta_old = beta\n",
    "        # update beta using gradient descent\n",
    "        beta = beta_old\n",
    "        # print objective value (so that you can monitor convergence)\n",
    "        obj = 0\n",
    "        print(obj)\n",
    "    \n",
    "    return beta"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}